package org.dlsu.mt.parser;

import java.util.StringTokenizer;
import java.util.Vector;

import org.dlsu.mt.DictionaryInterface;
import org.dlsu.mt.MorphInterface;
import org.dlsu.mt.parser.generator.ParseTable;

/**
 * A simple lexical analyzer that delimits by a space
 */
public class SimpleLexer implements LexicalAnalyzerInterface
{
	/** The list of tokens */
	Vector tokens;
	/** The marker for the current token*/
	int marker = 0;
	String delims;
	boolean returnDelim;
	DictionaryInterface dict;
	MorphInterface morph;

	/**
	 * Creates a new lexical analyzer given the string to parse.
	 * This lexer first parses the string and stores the tokens to a vector given the delimiter.
	 *
	 * @param delims A list of delimiters
	 * @param returnDelim Boolean specifying whether to consider the delimiter as a token or not. Set to true to consider, false otherwise.
	 */
	public SimpleLexer(String delims, boolean returnDelim)
	{
		this.delims = delims;
		this.returnDelim = returnDelim;
	}


	/**
	 * Sets the dictionary to be used by the lexical analyzer
	 *
	 * @param dict Dictionary to be used
	 */
	public void setDictionary(DictionaryInterface dict) {
		this.dict = dict;
	}
	/**
	 * Sets the morphological analyzer to be used by the lexical analyzer
	 *
	 * @param dict Morphological Analyzer to be used
	 */
	public void setMorph(MorphInterface morph) {
		this.morph = morph;
	}

	/**
	 * Returns a VectoredLexer after analyzing the given string.
	 *
	 * @param text The string to analyze/tokenize
	 * @return A VectoredLexer contaning the tokens
	 */
	public LexerInterface analyze(Object objText) {
		Word word;
		String token;
		
		
		String text = (String) objText;

		if (dict == null)
			throw new RuntimeException("Dictionary required for this lexical analyzer");
		
		tokens = new Vector();
		StringTokenizer st = new StringTokenizer(text,delims, returnDelim);
		//System.out.println("@SimpleLexer.java.analyze AHOOAHOO after tokenizer");
		while(st.hasMoreTokens())
		{
			//System.out.println("@SimpleLexer.java.analyze AHOOAHOO inside while");
			token = st.nextToken();			
			//System.out.println("@SimpleLexer.java.analyze AHOOAHOO token is " + token );
			if ( token.equals(ParseTable.ENDMARKER))
			{			
				word = new Word(ParseTable.ENDMARKER);
				word.addPOSTagEntry(ParseTable.ENDMARKER, null);
				tokens.addElement(word);
				//System.out.println("@SimpleLexer.java.analyze AHOOAHOO inside while1");
			}
			else if ( token.equals(ParseTable.EMPTY))
			{				
				word = new Word(ParseTable.EMPTY);
				word.addPOSTagEntry(ParseTable.EMPTY, null);
				tokens.addElement(word);
				//System.out.println("@SimpleLexer.java.analyze AHOOAHOO inside while2");
			}
			else
			{				
				word = dict.lookup(token);
				tokens.addElement(word);
				//System.out.println("@SimpleLexer.java.analyze AHOOAHOO inside while3");
			}
		}
		//System.out.println("@SimpleLexer.java.analyze AHOOAHOO outside while");
		return new VectoredLexer(tokens);
	}
}